#include <linux/gfp.h>
#include <linux/poll.h>
#include <asm/pgalloc.h>
+#include <asm/tlbflush.h>
#include <asm-xen/xen-public/io/blkif.h> /* for control ring. */
#include "blktap.h"
/* for poll: */
static wait_queue_head_t blktap_wait;
-/* Where things are inside the device mapping. */
-struct vm_area_struct *blktap_vma = NULL;
-unsigned long mmap_vstart;
-unsigned long rings_vstart;
-
/* Rings up to user space. */
static blkif_front_ring_t blktap_ufe_ring;
static blkif_back_ring_t blktap_ube_ring;
static int blktap_read_fe_ring(void);
static int blktap_read_be_ring(void);
+/* -------[ mmap region ]--------------------------------------------- */
+/*
+ * We use a big chunk of address space to map in-flight requests into,
+ * and export this region up to user-space. See the comments in blkback
+ * about this -- the two must be kept in sync if the tap is used as a
+ * passthrough.
+ */
+
+#define MAX_PENDING_REQS 64
+
+/* immediately before the mmap area, we have a bunch of pages reserved
+ * for shared memory rings.
+ */
+#define RING_PAGES 3 /* Ctrl, Front, and Back */
+
+/* Where things are inside the device mapping. */
+struct vm_area_struct *blktap_vma = NULL;
+unsigned long mmap_vstart; /* Kernel pages for mapping in data. */
+unsigned long rings_vstart; /* start of mmaped vma */
+unsigned long user_vstart; /* start of user mappings */
+
+#define MMAP_PAGES_PER_REQUEST \
+ (BLKIF_MAX_SEGMENTS_PER_REQUEST + 1)
+#define MMAP_PAGES \
+ (MAX_PENDING_REQS * MMAP_PAGES_PER_REQUEST)
+#define MMAP_VADDR(_start, _req,_seg) \
+ ( _start + \
+ ((_req) * MMAP_PAGES_PER_REQUEST * PAGE_SIZE) + \
+ ((_seg) * PAGE_SIZE))
+
+
+
+
/* -------[ blktap vm ops ]------------------------------------------- */
static struct page *blktap_nopage(struct vm_area_struct *vma,
if ( test_and_set_bit(0, &blktap_dev_inuse) )
return -EBUSY;
-
- printk(KERN_ALERT "blktap open.\n");
/* Allocate the ctrl ring. */
csring = (ctrl_sring_t *)get_zeroed_page(GFP_KERNEL);
blktap_dev_inuse = 0;
blktap_ring_ok = 0;
- printk(KERN_ALERT "blktap closed.\n");
+ DPRINTK(KERN_ALERT "blktap closed.\n");
/* Free the ring page. */
ClearPageReserved(virt_to_page(blktap_uctrl_ring.sring));
ClearPageReserved(virt_to_page(blktap_ube_ring.sring));
free_page((unsigned long) blktap_ube_ring.sring);
- /* Clear any active mappings. */
+ /* Clear any active mappings and free foreign map table */
if (blktap_vma != NULL) {
zap_page_range(blktap_vma, blktap_vma->vm_start,
blktap_vma->vm_end - blktap_vma->vm_start, NULL);
}
/* Note on mmap:
- * remap_pfn_range sets VM_IO on vma->vm_flags. In trying to make libaio
- * work to do direct page access from userspace, this ended up being a
- * problem. The bigger issue seems to be that there is no way to map
- * a foreign page in to user space and have the virtual address of that
- * page map sanely down to a mfn.
- * Removing the VM_IO flag results in a loop in get_user_pages, as
- * pfn_valid() always fails on a foreign page.
+ * We need to map pages to user space in a way that will allow the block
+ * subsystem set up direct IO to them. This couldn't be done before, because
+ * there isn't really a sane way to make a user virtual address down to a
+ * physical address when the page belongs to another domain.
+ *
+ * My first approach was to map the page in to kernel memory, add an entry
+ * for it in the physical frame list (using alloc_lomem_region as in blkback)
+ * and then attempt to map that page up to user space. This is disallowed
+ * by xen though, which realizes that we don't really own the machine frame
+ * underlying the physical page.
+ *
+ * The new approach is to provide explicit support for this in xen linux.
+ * The VMA now has a flag, VM_FOREIGN, to indicate that it contains pages
+ * mapped from other vms. vma->vm_private_data is set up as a mapping
+ * from pages to actual page structs. There is a new clause in get_user_pages
+ * that does the right thing for this sort of mapping.
+ *
+ * blktap_mmap sets up this mapping. Most of the real work is done in
+ * blktap_write_fe_ring below.
*/
static int blktap_mmap(struct file *filp, struct vm_area_struct *vma)
{
int size;
+ struct page **map;
+ int i;
- printk(KERN_ALERT "blktap mmap (%lx, %lx)\n",
+ DPRINTK(KERN_ALERT "blktap mmap (%lx, %lx)\n",
vma->vm_start, vma->vm_end);
+ vma->vm_flags |= VM_RESERVED;
vma->vm_ops = &blktap_vm_ops;
size = vma->vm_end - vma->vm_start;
}
size >>= PAGE_SHIFT;
- printk(KERN_INFO "blktap: 2 rings + %d pages.\n", size-1);
+ DPRINTK(KERN_INFO "blktap: 2 rings + %d pages.\n", size-1);
rings_vstart = vma->vm_start;
- mmap_vstart = rings_vstart + (RING_PAGES << PAGE_SHIFT);
+ user_vstart = rings_vstart + (RING_PAGES << PAGE_SHIFT);
/* Map the ring pages to the start of the region and reserve it. */
DPRINTK("Mapping ctrl_ring page %lx.\n", __pa(blktap_uctrl_ring.sring));
if (remap_pfn_range(vma, vma->vm_start,
__pa(blktap_uctrl_ring.sring) >> PAGE_SHIFT,
- PAGE_SIZE, vma->vm_page_prot)) {
- WPRINTK("ctrl_ring: remap_pfn_range failure!\n");
- }
+ PAGE_SIZE, vma->vm_page_prot))
+ goto fail;
DPRINTK("Mapping be_ring page %lx.\n", __pa(blktap_ube_ring.sring));
if (remap_pfn_range(vma, vma->vm_start + PAGE_SIZE,
__pa(blktap_ube_ring.sring) >> PAGE_SHIFT,
- PAGE_SIZE, vma->vm_page_prot)) {
- WPRINTK("be_ring: remap_pfn_range failure!\n");
- }
+ PAGE_SIZE, vma->vm_page_prot))
+ goto fail;
DPRINTK("Mapping fe_ring page %lx.\n", __pa(blktap_ufe_ring.sring));
if (remap_pfn_range(vma, vma->vm_start + ( 2 * PAGE_SIZE ),
__pa(blktap_ufe_ring.sring) >> PAGE_SHIFT,
- PAGE_SIZE, vma->vm_page_prot)) {
- WPRINTK("fe_ring: remap_pfn_range failure!\n");
- }
-
+ PAGE_SIZE, vma->vm_page_prot))
+ goto fail;
+
+ /* Mark this VM as containing foreign pages, and set up mappings. */
+ map = kmalloc(((vma->vm_end - vma->vm_start) >> PAGE_SHIFT)
+ * sizeof(struct page_struct*),
+ GFP_KERNEL);
+ if (map == NULL) goto fail;
+
+ for (i=0; i<((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); i++)
+ map[i] = NULL;
+
+ vma->vm_private_data = map;
+ vma->vm_flags |= VM_FOREIGN;
+
blktap_vma = vma;
blktap_ring_ok = 1;
return 0;
+ fail:
+ /* Clear any active mappings. */
+ zap_page_range(vma, vma->vm_start,
+ vma->vm_end - vma->vm_start, NULL);
+
+ return -ENOMEM;
}
static int blktap_ioctl(struct inode *inode, struct file *filp,
RING_HAS_UNPUSHED_REQUESTS(&blktap_ufe_ring) ||
RING_HAS_UNPUSHED_RESPONSES(&blktap_ube_ring) ) {
+ flush_tlb_all();
+
RING_PUSH_REQUESTS(&blktap_uctrl_ring);
RING_PUSH_REQUESTS(&blktap_ufe_ring);
RING_PUSH_RESPONSES(&blktap_ube_ring);
/*-----[ Data to/from user space ]----------------------------------------*/
+static void fast_flush_area(int idx, int nr_pages)
+{
+ multicall_entry_t mcl[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+ int i;
+
+ for ( i = 0; i < nr_pages; i++ )
+ {
+ MULTI_update_va_mapping(mcl+i, MMAP_VADDR(mmap_vstart, idx, i),
+ __pte(0), 0);
+ }
+
+ mcl[nr_pages-1].args[MULTI_UVMFLAGS_INDEX] = UVMF_TLB_FLUSH|UVMF_ALL;
+ if ( unlikely(HYPERVISOR_multicall(mcl, nr_pages) != 0) )
+ BUG();
+}
+
+
+extern int __direct_remap_area_pages(struct mm_struct *mm,
+ unsigned long address,
+ unsigned long size,
+ mmu_update_t *v);
+
int blktap_write_fe_ring(blkif_request_t *req)
{
blkif_request_t *target;
- int error, i;
+ int i;
+ unsigned long remap_prot;
+ multicall_entry_t mcl[BLKIF_MAX_SEGMENTS_PER_REQUEST+1];
+ mmu_update_t mmu[BLKIF_MAX_SEGMENTS_PER_REQUEST];
/*
* This is called to pass a request from the real frontend domain's
return 0;
}
- target = RING_GET_REQUEST(&blktap_ufe_ring,
- blktap_ufe_ring.req_prod_pvt);
+ remap_prot = _PAGE_PRESENT|_PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_RW;
+ flush_cache_all(); /* a noop on intel... */
+
+ target = RING_GET_REQUEST(&blktap_ufe_ring, blktap_ufe_ring.req_prod_pvt);
memcpy(target, req, sizeof(*req));
- /* Attempt to map the foreign pages directly in to the application */
+ /* Map the foreign pages directly in to the application */
for (i=0; i<target->nr_segments; i++) {
+ unsigned long buf;
+ unsigned long uvaddr;
+ unsigned long kvaddr;
+ unsigned long offset;
+
+ buf = target->frame_and_sects[i] & PAGE_MASK;
+ uvaddr = MMAP_VADDR(user_vstart, ID_TO_IDX(req->id), i);
+ kvaddr = MMAP_VADDR(mmap_vstart, ID_TO_IDX(req->id), i);
+
+ MULTI_update_va_mapping_otherdomain(
+ mcl+i,
+ kvaddr,
+ pfn_pte_ma(buf >> PAGE_SHIFT, __pgprot(remap_prot)),
+ 0,
+ ID_TO_DOM(req->id));
+
+ phys_to_machine_mapping[__pa(kvaddr)>>PAGE_SHIFT] =
+ FOREIGN_FRAME(buf >> PAGE_SHIFT);
+
+ __direct_remap_area_pages(blktap_vma->vm_mm,
+ uvaddr,
+ PAGE_SIZE,
+ &mmu[i]);
+ mmu[i].val = (target->frame_and_sects[i] & PAGE_MASK)
+ | pgprot_val(blktap_vma->vm_page_prot);
+
+ offset = (uvaddr - blktap_vma->vm_start) >> PAGE_SHIFT;
+ ((struct page **)blktap_vma->vm_private_data)[offset] =
+ pfn_to_page(__pa(kvaddr) >> PAGE_SHIFT);
+ }
+
+ /* Add the mmu_update call. */
+ mcl[i].op = __HYPERVISOR_mmu_update;
+ mcl[i].args[0] = (unsigned long)mmu;
+ mcl[i].args[1] = target->nr_segments;
+ mcl[i].args[2] = 0;
+ mcl[i].args[3] = ID_TO_DOM(req->id);
- error = direct_remap_area_pages(blktap_vma->vm_mm,
- MMAP_VADDR(ID_TO_IDX(req->id), i),
- target->frame_and_sects[i] & PAGE_MASK,
- PAGE_SIZE,
- blktap_vma->vm_page_prot,
- ID_TO_DOM(req->id));
- if ( error != 0 ) {
- printk(KERN_INFO "remapping attached page failed! (%d)\n", error);
- /* the request is now dropped on the floor. */
- return 0;
+ BUG_ON(HYPERVISOR_multicall(mcl, target->nr_segments+1) != 0);
+
+ /* Make sure it all worked. */
+ for ( i = 0; i < target->nr_segments; i++ )
+ {
+ if ( unlikely(mcl[i].result != 0) )
+ {
+ DPRINTK("invalid buffer -- could not remap it\n");
+ fast_flush_area(ID_TO_IDX(req->id), target->nr_segments);
+ return -1;
}
}
-
+ if ( unlikely(mcl[i].result != 0) )
+ {
+ DPRINTK("direct remapping of pages to /dev/blktap failed.\n");
+ return -1;
+ }
+
+
+ /* Mark mapped pages as reserved: */
+ for ( i = 0; i < target->nr_segments; i++ )
+ {
+ unsigned long kvaddr;
+
+ kvaddr = MMAP_VADDR(mmap_vstart, ID_TO_IDX(req->id), i);
+ SetPageReserved(pfn_to_page(__pa(kvaddr) >> PAGE_SHIFT));
+ }
+
+
blktap_ufe_ring.req_prod_pvt++;
return 0;
{
/* This is called to read responses from the UFE ring. */
- RING_IDX i, rp;
+ RING_IDX i, j, rp;
blkif_response_t *resp_s;
blkif_t *blkif;
active_req_t *ar;
DPRINTK("resp->fe_ring\n");
ar = lookup_active_req(ID_TO_IDX(resp_s->id));
blkif = ar->blkif;
- zap_page_range(blktap_vma, MMAP_VADDR(ID_TO_IDX(resp_s->id), 0),
+ for (j = 0; j < ar->nr_pages; j++) {
+ unsigned long vaddr;
+ struct page **map = blktap_vma->vm_private_data;
+ int offset;
+
+ vaddr = MMAP_VADDR(user_vstart, ID_TO_IDX(resp_s->id), j);
+ offset = (vaddr - blktap_vma->vm_start) >> PAGE_SHIFT;
+
+ ClearPageReserved(virt_to_page(vaddr));
+ map[offset] = NULL;
+ }
+
+
+ zap_page_range(blktap_vma,
+ MMAP_VADDR(user_vstart, ID_TO_IDX(resp_s->id), 0),
ar->nr_pages << PAGE_SHIFT, NULL);
+ fast_flush_area(ID_TO_IDX(resp_s->id), ar->nr_pages);
write_resp_to_fe_ring(blkif, resp_s);
blktap_ufe_ring.rsp_cons = i + 1;
kick_fe_domain(blkif);
{
int err;
+ if ( (mmap_vstart = allocate_empty_lowmem_region(MMAP_PAGES)) == 0 )
+ BUG();
+
err = misc_register(&blktap_miscdev);
if ( err != 0 )
{
projects / xen.git / commitdiff